{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from dotenv import load_dotenv\n",
    "import os\n",
    "from langchain.chains import LLMChain\n",
    "from langchain.llms import OpenAI\n",
    "\n",
    "# Laden Sie die Umgebungsvariablen aus der .env-Datei\n",
    "load_dotenv()\n",
    "API_KEY = os.environ.get(\"API_KEY\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.prompts.prompt import PromptTemplate\n",
    "\n",
    "llm = OpenAI(temperature=0)\n",
    "\n",
    "_PROMPT_TEMPLATE = \"\"\"You are an expert professor specialized in grading students' answers to questions.\n",
    "You are grading the following question:\n",
    "{query}\n",
    "Here is the real answer:\n",
    "{answer}\n",
    "You are grading the following predicted answer:\n",
    "{result}\n",
    "What grade do you give from 0 to 10, where 0 is the lowest (very low similarity) and 10 is the highest (very high similarity)?\n",
    "\"\"\"\n",
    "\n",
    "PROMPT = PromptTemplate(\n",
    "    input_variables=[\"query\", \"answer\", \"result\"], template=_PROMPT_TEMPLATE\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "context_examples = [\n",
    "    {\n",
    "        \"question\": \"Do you offer vegetarian or vegan options?\",\n",
    "        \"context\": \"Yes, we have a range of dishes to cater to vegetarians and vegans\",\n",
    "    },\n",
    "    {\n",
    "        \"question\": \"What are the hours of operation for your restaurant?\",\n",
    "        \"context\": \"Our restaurant is open from 11 a.m. to 10 p.m. from Monday to Saturday. On Sundays, we open at 12 p.m. and close at 9 p.m.\",\n",
    "    },\n",
    "]\n",
    "QA_PROMPT = \"Answer the question based on the  context\\nContext:{context}\\nQuestion:{question}\\nAnswer:\"\n",
    "template = PromptTemplate(input_variables=[\"context\", \"question\"], template=QA_PROMPT)\n",
    "qa_chain = LLMChain(llm=llm, prompt=template)\n",
    "predictions = qa_chain.apply(context_examples)\n",
    "predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.evaluation.qa import ContextQAEvalChain\n",
    "\n",
    "eval_chain = ContextQAEvalChain.from_llm(llm)\n",
    "graded_outputs = eval_chain.evaluate(\n",
    "    context_examples, predictions, question_key=\"question\", prediction_key=\"text\"\n",
    ")\n",
    "graded_outputs"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "autogpt",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
